from urllib import request,error
import re,time,json

def getPage(url):
    try:
        #定义请求头
        headers = {
            'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',
            }

        #封装请求对象
        req = request.Request(url,headers=headers)

        #执行爬去
        res = request.urlopen(req)

        #判断 响应状态，并响应爬去内容
        if res.code == 200:
            return res.read().decode('utf-8')
        else:
            print(res.code)
            return None
    except error.URLError:
        print("错啦！！")
        return None

def parsePage(html):
    '''解析爬取网页的内容，并返回字段结果'''
    #定义正则表达式
    pat = '<i class="board-index board-index-[0-9]+">([0-9]+)</i>.*?<img data-src="(.*?)" alt="(.*?)" class="board-img" />.*?<p class="star">(.*?)</p>.*?<p class="releasetime">(.*?)</p>.*?<i class="integer">([0-9\.]+)</i><i class="fraction">([0-9]+)</i>'
    
    #执行解析
    items = re.findall(pat,html,re.S)

    #遍历封装数据并返回
    for item in items:
        yield{
            'index':item[0],
            'title':item[2],
            'iamge':item[1],
            'actor':item[3].strip(),
            'time':item[4].strip(),
            'score':item[5]+item[6],
            }
        
def writeFile(content):
    '''文件追加写操作'''
    try:
        with open("./result.txt",'a',encoding='utf-8') as f:
            f.write(json.dumps(content,ensure_ascii=False)+"\n")
    except error as e :
        pritn("写入错误！")

def main(offset):
    '''主程序函数，负责调度执行爬虫处理'''
    url = 'http://maoyan.com/board/4?offset=' + str(offset)
    #print(url)
    html = getPage(url)

    #判断是否爬取到数据，并调度执行爬虫处理
    if html:
        #print("1111111")
        for item in parsePage(html):
            writeFile(item)
    else:
        print("空！！！")
            
if __name__ == '__main__':
    for i in range(10):
        main(offset=i*10)
        time.sleep(1)



    
